#!/usr/bin/env python2.5

## Revelator Module EIItoCNL
## Version 1.0
## Written by Josh "Lain" Johnson, 2 August 2009
## Modification 1:  14 September 2009
## All lefts reserved.

import nltk, sys, re
from nltk import data, grammar, tag, corpus
from nltk.data import path
from nltk.parse import EarleyChartParser
from nltk.tag import crf
from nltk.corpus import wordnet
path += ['.']
taggerPath = '/NLP_TOOLS/pos_taggers/postagger/latest'
path += taggerPath

## Self-developed modules:
# import hmmtagger
# import preterms
import seqtagger
import tokenize
import prompt
import define
import extract
import commands
import getnps

class EIItoCNL:
	"""This will parse a sentence according to a grammar supplied via a gold standard.  If there is a structural 
		ambiguity, which occurs when a sentence has more than one distinct parse, then a customized error message
		is output to the user.  Otherwise, the sentence is converted into a Controlled Natural Language (CNL) format."""

	def __init__(self, for_IM = False, TRON = 1):
		"""Initialize the grammar by first loading in the file of acceptible Evidence Implies Inference (EII) format parses,
			then reading in and reformatting the input sentence, before testing its acceptibility and printing output
			accordingly."""

		## Zerost, set the file and parameter defaults.
		self.tagfile = "sentences.dat"
		self.grammar = "grammar.cfg"
		self.promptings = "allambis.prmz"
		self.nolines = 0
		self.N = 4
		self.old_topic = "Potpourri"
		self.new_topic = "Potpourri"
		self.noun_tags = ["NN", "NNS", "FW"]
		
		## These correspond to list indices for the resolution process.
		self.resolvingEAmbi = 0
		self.resolvingIAmbi = 0
		self.resolvingDef = 0

		## -1 if not resolved, 0 if one step from being resolved, and 1 if resolved
		self.inference_resolved = 1
		self.evidence_resolved = 1
		self.resolved = 1

		## Flag for whether this is the initial run, used for resolving word definitions.
		initial = True
		altered = dict()
		ideal = dict()
		words = dict()

		## First, read in the grammar file.
		str = ''
		gramfile = open(self.grammar, 'r')
		for line in gramfile.readlines():
			str += line
			gramfile.close()
		
		## Then, initialize the tagger.
		# HMMT = hmmtagger.getHMMTagger(self.tagfile, self.nolines)
		# self.HMMTagger = HMMT.trainHMM()
		(self.SeqTagger, self.train_data) = seqtagger.getTagger(self.tagfile, self.nolines, self.N)
		if (TRON > 1):	print "This tagger is %0.4f%% accurate." % self.score()

		## Next, we will read in the ambiguous sentence prompts.
		self.messages = prompt.init_messages(self.promptings)

		received = ""
		playing = True
		while (playing):

			## After that, read in the play or commands from the user.  For now, we will take only one at a time.  
			## TODO 1:  Adapt for multiple sentences per input.  Consider batch_parse and the like.
			## We will keep taking input until the user wishes to quit.
			if (self.resolved != 0):
				received = self.getPlay(self.promptText)

				## Skip the rest of stuff if the user just types in a command.
				ternary = commands.aCommand(received)
				if (ternary != 0):
					print
					if (ternary == -1):	playing = False
					continue

			## And this lets us self-define words.
			# (received, words, altered) = define.paran(received, words, altered, ideal)

			## We can self-define words in this part of the code.
			## N. B. that we will tokenize and tag the input twice, once before and once after, so that we can extract
			##   the paranthetical definitiions and not have them affect the final tagging.
			if (self.resolved == 1):
				(words, altered, allNPs, received) = self.call_paranNP(received, words, altered, ideal)

				## Bail-out code:
				# playing = False
				# print received
				# continue

				## Add the terminals to the grammar.  We will parse the if- and then-clauses separately.
				(if_clause, then_clause, str, pos_if, pos_then) = self.listen(str, received)

				## This gets rid of unnecessary punctuation.
				# if_clause = self.strip_punc(if_clause)
				# then_clause = self.strip_punc(then_clause)

				## Now, we get to the grammar.
				EII_grammar = grammar.parse_cfg(str)
				self.EII_Earley = EarleyChartParser(EII_grammar, trace = TRON)

				## Following that, we (re)define common nouns.
				(altered, ideal, words) = self.get_defs(if_clause.split(), pos_if.split(), altered, ideal, words, initial, allNPs)
				(altered, ideal, words) = self.get_defs(then_clause.split(), pos_then.split(), altered, ideal, words, initial, allNPs)

				## Verify the defintions (debug only).
				# for NP in allNPs.values():
				#	print "NP", NP, "is defined as", words[NP], "."

				## Then, test this sentence against our grammar.
				## IF clause:
				valid_if = self.parse(if_clause, pos_if, "evidence")
				## THEN clause:
				valid_then = self.parse(then_clause, pos_then, "inference")
				initial = False

				## If valid, print a copy of the evidence and inference to STDOUT.  Otherwise, tell the user what was wrong.
				if (valid_if and valid_then):
					self.printEI(if_clause, then_clause, altered, words, allNPs)
				elif (valid_if):
					print "I was able to understand your evidential statement, but your inferential statement did not parse.  Could you please restate\nyour entire play?"
				else:
					print "I was able to understand your statement of inference, but your statement of evidence did not parse.  Could you please restate\nyour entire play?"

				## Reset defaults.
				self.promptText = "\tUSER>  "
				self.resolved = 1
				self.evidence_resolved = 1
				self.inference_resolved = 1
			if (self.evidence_resolved != 1):
				valid_if = self.parse(received, pos_if, "evidence")
			if (self.inference_resolved != 1):
				valid_then = self.parse(received, pos_then, "inference")


	def getPlay(self, prompt):
		"""Gets the play from the user."""
		return (raw_input(prompt))

	def listen(self, grammar_string, play):
		"""Runs the input play from the user and runs it through a POS tagger."""

		## This gets rid of unnecessary punctuation then tokenize input from the user.
		# sent = tokenize.tokenize(self.strip_punc(raw_input(" USER>\t")), 0)
		sent = tokenize.tokenize(play, 0)
		print
		
		## Default POS tagger:  Stanford POS tagger
		## TODO 2:  Use the Stanford tagger.
		## Backup tagger:  Mallet CRF tagger.
		# MT = crf.MalletCRF('/home/lain0523/tmp/Rev/test_model_1')
		# tagging = MT.tag(sent.split())
		
		## Second backup tagger:  HMM tagger.
		# tagging = self.HMMTagger.tag(sent.split())
       
		## Third backup tagger:  Sequential tagger.
		##   (ASIDE:  The number of taggers is getting ridiculous.)

		## We are parsing the if-clause and then-clause separately.

		(ifc, thenc) = extract.getEI(sent)
		# print ifc, thenc
		tagging_if = self.SeqTagger.tag(ifc.split())
		tagging_then = self.SeqTagger.tag(thenc.split())
		tags_if = ""
		tags_then = ""

		## Add the terminals to our grammar string:
		for (token, tag) in tagging_if:
			term = tag + ' -> "' + token + '"\n'
			tags_if += tag + " "
			# print "Tagged", term[:-1]
			grammar_string += term
			# print "Adding", tag, '-> "' + token + '" to the grammar.'
		for (token, tag) in tagging_then:
			term = tag + ' -> "' + token + '"\n'
			tags_then += tag + " "
			# print "Tagged", term[:-1]
			grammar_string += term
			# print "Adding", tag, '-> "' + token + '" to the grammar.'
		return (ifc, thenc, grammar_string, tags_if[:-1], tags_then[:-1])

	def no_parses(self, EorI):
		"""Prints an error message if the translator cannot make any sense of the sentence."""

		print "TUTOR>  I was unable to parse your", EorI + ".  Please rephrase it for me, so that it is easier to understand.\n"

	def mult_parses(self, EorI):
		"""Prints an error message in case we have structural ambiguity."""
		
		print "TUTOR>  There were many ways of understanding the " + EorI + " you typed.\n";

	def EIItoCNL(self, all_parses):
		"""Converts the EII-formed string into the CNL format."""

		## TODO 4:  After getting the form from Arun, find a way to convert the EII string to CNL.
		return all_parses
	
	def score(self):
		"""Give an accuracy score (on a [0, 100] scale)."""
		return 100.0 * self.SeqTagger.evaluate(self.train_data)

	def parse(self, clause, tag_seq, EorI):
		"""Verify that the clauses parse."""
		if (EorI == "evidence"):	index = self.resolvingEAmbi
		else:	index = self.resolvingIAmbi
		if (self.for_IM):	poss = 2
		else:
			parses = self.EII_Earley.nbest_parse(clause.split())
			# print pos_seq
			poss = len(parses)
		if (poss == 0):

			## We can't parse this sentence using our grammar.
			self.no_parses(EorI)
			return False
		if (poss > 1):

			## We have multiple parses.  First, we try and use our templates to resolve ambiguities.
			self.mult_parses(EorI)
			# print parses[0], "\n"
			if (tag_seq not in self.messages):

				## We don't have a template, but we will let this clause pass anyway.
				print "I have no template to resolve the", EorI + ".  We'll leave it as is, then."
			else:	

				## We here prompt the user to confirm one of various readings of the clause.
				if (self.for_IM):
					if (index > -1):
							
						## States that we are in the midst of resolving the structural ambiguity.
						self.promptText = prompt.fitInto(clause, self.messages[tag_seq][index]) + "  "
						if (define.yes(clause)):
							# print "Going with template", template
							chosen = True
							exec ("self.%s_resolved = 0" % EorI)
						else:	
							index += 1
							exec ("self.%s_resolved = -1" % EorI)
				else:
					chosen = False
					for template in self.messages[tag_seq]:
						yn = raw_input(prompt.fitInto(clause, template) + "  ")
						if (define.yes(yn)):
							print "Going with template", template
							chosen = True
							break			
				if ((not chosen) or (self.for_IM and (len(self.messages[tag_seq]) > index))):

					## The user has not chosen one, which means we're confused.  Let's have him rephrase it, then.
					print "Could you please rephrase your", EorI + ", since I can't understand your intended meaning?"
					self.resolvingAmbi = 0
					exec ("self.%s_resolved = 1" % EorI)
					if (EorI == "evidence"):	self.resolvingEAmbi = index
					else:	self.resolvingIAmbi = index
					return False
		else:	# (poss == 1)

			## This is acceptible, so we print the output as a CNL string.
			asCNL = self.EIItoCNL(parses)
			print "REVELATOR> ", asCNL
			return True

	def get_defs(self, tokens, pos, alt, best, lex, first, NPdict):
		"""(Re)Define the words in the if/then clause provided."""
		if (len(tokens) != len(pos)):	
			print "Error:  The tokenized sentence doesn't have fully-matching POS tags."
			return (alt, best, lex)
		if (first):
			for np in NPdict.values():
				if (np not in lex):
					alt[np] = True
					if (np not in best):	best[np] = 1
					try:

						## This last loop assumes a plural noun ending in 'ies', looking up the base word accordingly.
						try:

							## This one assumes it's a plural noun ending in 'es', and looks up the base word.
							try:

								## This one assumes that, if it ends in 's', it's a plural noun and looks up the base word.
								try:

									## This loop tries just the word itself (executed first).
									lex[np] = wordnet.synset(np + ".n." + str(best[np])).definition
								except (nltk.corpus.reader.wordnet.WordNetError):
									if (np[-1] == "s"):	lex[np] = wordnet.synset(np[:-1] + ".n." + str(best[np])).definition
							except (nltk.corpus.reader.wordnet.WordNetError):
								if (np[-2:] == "es"):	lex[np] = wordnet.synset(np[:-2] + ".n." + str(best[np])).definition
						except (nltk.corpus.reader.wordnet.WordNetError):
							if (np[-3:] == "ies"):	lex[np] = wordnet.synset(np[:-3] + "y.n." + str(best[np])).definition
					except (nltk.corpus.reader.wordnet.WordNetError):
						(alt[np], lex[np]) = define.define(np, "", best[np], False, self.old_topic, self.new_topic)
			# for x in range(len(tokens)):
			#	word = tokens[x]
			#	tag = pos[x]
			#	if ((tag in self.noun_tags) and (word not in lex)):
			#		alt[word] = True
			#		if (word not in best):	best[word] = 1
			#		lookup = word + ".n." + str(best[word])
			#		try:
			#			lex[word] = wordnet.synset(lookup).definition
			#		except(Exception):
			#			(alt[word], lex[word]) = define.define(word, "", best[word], False, self.old_topic, self.new_topic)
			return (alt, best, lex)
		if (self.old_topic == self.new_topic):
			alt = dict()
		for np in NPdict.values():
			if (np not in lex):
				alt[np] = True
				if (np not in best):	best[np] = 1
				try:
					try:
						try:
							try:
								lex[np] = wordnet.synset(np + ".n." + str(best[np])).definition
							except (nltk.corpus.reader.wordnet.WordNetError):
								if (np[-1] == "s"):	lex[np] = wordnet.synset(np[:-1] + ".n." + str(best[np])).definition
						except (nltk.corpus.reader.wordnet.WordNetError):
							if (np[-2:] == "es"):	lex[np] = wordnet.synset(np[:-2] + ".n." + str(best[np])).definition
					except (nltk.corpus.reader.wordnet.WordNetError):
						if (np[-3:] == "ies"):	lex[np] = wordnet.synset(np[:-3] + "y.n." + str(best[np])).definition
				except (nltk.corpus.reader.wordnet.WordNetError):
					(alt[np], lex[np]) = define.define(np, "", best[np], True, self.old_topic, self.new_topic)
		# for x in range(len(tokens)):
		#	word = tokens[x]
		#	tag = pos[x]
		#	if ((tag in self.noun_tags) and (word not in lex)):
		#		(alt[word], lex[word]) = define.define(word, lex[word], best[word], True, self.old_topic, self.new_topic)
		return (alt, best, lex)

	def printEI(self, ifc, thenc, alt, lex, NPdict):
		"""Prints the evidence and inference, with (newly-/un-)defined words."""
		# output = ""
		print "EVIDENCE:\t",
		# for word in ifc.split():
			# output += word + " "
			# if (word in alt):	output += "(" + lex[word] + ") "
		# print output
		# print tokenize.tokenize(output, -1)
		temp = ifc
		for NP in NPdict.values():
			temp = re.sub(NP, NP + " (" + lex[NP] + ")", temp)
		print temp
		print "\nINFERENCE:\t",
		temp = thenc
		for NP in NPdict.values():
			temp = re.sub(NP, NP + " (" + lex[NP] + ")", temp)
		print temp
		# output = ""
		# for word in thenc.split():
			# output += word + " "
			# if (word in alt):	output += "(" + lex[word] + ") "
		# print output
		# print tokenize.tokenize(output, -1)
		print

	def strip_punc(self, str):
		"""Strips the punctuation (esp. periods, commas, and semicolons) from the rear of the clauses."""
		if (str.strip()[-1] in [",", ".", ";"]):	str = str.strip()[:-1]
		return str

	def call_paranNP(self, str, lex, alt, best):
		"""Does the tokenizing, tagging, and substitution necessary before calling define.paranNP(...)."""
		noparans = re.sub(r" ?\(([^)]*)\)", "", str)
		prelim = tokenize.tokenize(noparans, 0)
		prelim_tagging = self.SeqTagger.tag(prelim.split())
		tokens = list()
		tagging = list()
		for (w, t) in prelim_tagging:
			tokens.append(w)
			tagging.append(t)
		allNPs = getnps.NPs(tokens, tagging)
		(lex, alt) = define.paranNP(allNPs, str, lex, alt, best)
		return (lex, alt, allNPs, noparans)


## Main module
if (__name__ == "__main__"):
	print "Welcome to Revelator!\n"
	EIItoCNL()
	print "Have a nice day!  Come play again sometime soon!\n"
